/**
 * Created by IntelliJ IDEA.
 * User: tobias
 * Date: 5/5/11
 * Time: 4:56 PM
 * To change this template use File | Settings | File Templates.
 */

import edu.udo.cs.wvtool.config.WVTConfiguration;
import edu.udo.cs.wvtool.config.WVTConfigurationFact;
import edu.udo.cs.wvtool.main.WVTDocumentInfo;
import edu.udo.cs.wvtool.main.WVTFileInputList;
import edu.udo.cs.wvtool.main.WVTool;
import edu.udo.cs.wvtool.util.WVToolException;
import edu.udo.cs.wvtool.wordlist.WVTWordList;

import java.io.File;
import java.io.IOException;

import static net.sf.javaml.tools.data.FileHandler.loadSparseDataset;

public class BaseModule  {

    private WVTFileInputList filelist;
    private WVTool tool = new WVTool(false);
    private DataSet data = new DataSet();
    private WVTConfiguration config = new WVTConfiguration();

    public void parseDocuments(String directory) throws Exception{
        File dir = new File(directory);
        if(!dir.isDirectory()) {
            throw new Exception("not a directory");
        }
        File[] subdirs = dir.listFiles();
        filelist = new WVTFileInputList(subdirs.length);
        for(int i = 0; i < subdirs.length; i++) {
            filelist.addEntry(
                    new WVTDocumentInfo(subdirs[i].toString(), "txt", "", "english", i));
        }
    }
    public WVTWordList createWordList() throws WVToolException {
        return tool.createWordList(filelist, config);
    }
    public DataSet createVectors(WVTWordList wordlist) throws Exception {
        wordlist.pruneByFrequency(20, 300);
        System.out.println(wordlist.getNumWords());
        config.setConfigurationRule(
			WVTConfiguration.STEP_OUTPUT,
			new WVTConfigurationFact(data));

        tool.createVectors(filelist, config, wordlist);
        return data;
    }
    public DataSet loadDataSet(File file) throws IOException {
        return new DataSet(loadSparseDataset(file, 0, "\t", ":"));
    }
}
